#coding=utf-8
#---------------------------------------------------
#导入库
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.model_selection import cross_val_score, KFold
import xgboost as xgb
#----------------数据探索----------------
#数据预处理
#加载训练集
train_df = pd.read_csv('./基于论文摘要的文本分类与查询性问答公开数据/train.csv', sep=',')
#加载测试集
test_df = pd.read_csv('./基于论文摘要的文本分类与查询性问答公开数据/test.csv', sep=',')

#EDA数据探索性分析
train_df.head()

test_df.head()

#----------------特征工程----------------
#将Topic(Label)编码
train_df['Topic(Label)'], lbl = pd.factorize(train_df['Topic(Label)'])

#将论文的标题与摘要组合为 text 特征
train_df['Title'] = train_df['Title'].apply(lambda x: x.strip())
train_df['Abstract'] = train_df['Abstract'].fillna('').apply(lambda x: x.strip())
train_df['text'] = train_df['Title'] + ' ' + train_df['Abstract']
train_df['text'] = train_df['text'].str.lower()

test_df['Title'] = test_df['Title'].apply(lambda x: x.strip())
test_df['Abstract'] = test_df['Abstract'].fillna('').apply(lambda x: x.strip())
test_df['text'] = test_df['Title'] + ' ' + test_df['Abstract']
test_df['text'] = test_df['text'].str.lower()

#使用tfidf算法做文本特征提取
# tfidf = TfidfVectorizer(max_features=2500)
#使用Count Vectorizer来fit训练集和测试集
ctv = CountVectorizer()

#----------------模型训练----------------

train_tfidf = ctv.fit_transform(train_df['text'])
test_tfidf = ctv.transform(test_df['text'])

# clf = SGDClassifier()
clf = xgb.XGBClassifier(max_depth=7, n_estimators=200, colsample_bytree=0.8,
                        subsample=0.8, nthread=10, learning_rate=0.1)
cvs = cross_val_score(clf, train_tfidf, train_df['Topic(Label)'], cv=5)
#用于获取每个交叉验证的得分,然后根据得分score来选择合适的超参数,通常需要编写手动完成交叉
print(cvs)

clf.fit(train_tfidf, train_df['Topic(Label)'])
test_df['Topic(Label)'] = clf.predict(test_tfidf)


#----------------结果输出----------------
print(test_df['Topic(Label)'])
test_df['Topic(Label)'] = test_df['Topic(Label)'].apply(lambda x: lbl[x])
test_df[['Topic(Label)']].to_csv('submit-CountVectorizer.csv', index=None)